package src;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.Vector;

import lucene.LuceneConstants;

/**
 * @author Govind Kothari
 **/

public class GetDocs {

	static boolean map = true;
	public static Vector getDocuments(Vector matrix, String indexDir, 
			Hashtable idfHash){
		
		SearchFaq sf = new SearchFaq(indexDir);
		Vector allDocs = null;
		int [] frontierIndex = new int [matrix.size()];
		for(int i=0; i< matrix.size(); i++){
			frontierIndex[i]=0;
		}
		double upperBound = getUpperBound(matrix,frontierIndex);
		double lowerBound = 0;
		int maxWeightWordIndex;
	
		while(lowerBound < upperBound){	
			maxWeightWordIndex = getMaxWeightWordIndex(matrix,frontierIndex);
			Variation var = null;
			if( maxWeightWordIndex != -1){
				var = (Variation)((Vector)matrix.get(maxWeightWordIndex))
				.get(frontierIndex[maxWeightWordIndex]);
				frontierIndex[maxWeightWordIndex]++;
			} else{
				break;
			}
			//think about bugs
			
			Vector docs = sf.getHitIds(var.word);	
			allDocs = Union(allDocs, docs, var, matrix.size());			
			lowerBound = getLowerBound(allDocs);			
			upperBound = getUpperBound(matrix,frontierIndex);
		}
		
		try {
			allDocs = getCompleteScore(allDocs,matrix,frontierIndex,sf, idfHash);
		} catch (UnsupportedEncodingException e) {
			e.printStackTrace();
		}
	
		//allDocs = normalizeQuestions(allDocs);
		if(allDocs != null){
			Collections.sort(allDocs, new ScoreComparer());
		}
		return allDocs;
	}

	public static Vector naiveGetDocuments(Vector matrix, String indexDir){
		
		SearchFaq sf = new SearchFaq(indexDir);
		Vector allDocs = null;

		for(int i=0; i< matrix.size(); i++){
			Vector wordNVariation = (Vector)matrix.get(i);
			for(int j=0; j< wordNVariation.size(); j++){
				Variation v = (Variation)wordNVariation.get(j);
				Vector docs = sf.getHitIds(v.word);
				allDocs = Union(allDocs, docs, v, matrix.size());
			}
		}		
		if(allDocs != null){
			Collections.sort(allDocs, new ScoreComparer());
		}
		return allDocs;
	}
	
	public static Vector getCompleteScore(Vector allDocs, Vector matrix, 
			int [] frontierIndex, SearchFaq sf, Hashtable idfHash)
	throws UnsupportedEncodingException{

		if(allDocs == null){
			Debug.print("getCompleteScore returned NULL");
			return null;
		}

		for(int docnum=0;docnum< allDocs.size();docnum++){
			boolean flag = false;
			MyDoc md = (MyDoc)allDocs.get(docnum);
			QApair qap = sf.getDocFromSingleId(md.hitId);
			md.setQApair(qap);			
			String question = ((QApair)md.qap).quest;
			question = question.trim();
			question = question.replaceAll("\\b\\s{2,}\\b", " ");
			String [] words = question.split("[ ]+");


			
			for(int tokenIndex=0; tokenIndex< matrix.size(); tokenIndex++){
				double maxWeight = 0;
				
				if(!md.isIndexSet(tokenIndex)){
					Vector wordNvariation = (Vector)matrix.get(tokenIndex);
					for(int j=0; j< words.length; j++){											
						boolean check = false;
						
						for(int k = 0; k < wordNvariation.size(); k++){							
							Variation v = (Variation)wordNvariation.get(k);							
							String token = v.word;
							if(token.equals(words[j])){
								//Debug.printUTF(question);
								//Debug.printUTF(token);
								maxWeight = v.weight;
								if(maxWeight>0.1&&md.checkIfNotAdded(words[j])){
									md.incrementCurrentScore(maxWeight);
									md.setIndex(tokenIndex);
									md.addWord(words[j],maxWeight);
									check = true;
									break;
								}	
							}
							if(check){break;}
						}
					}
				}	
			}	
		}	
		return allDocs;
	}

	public static Vector getCompleteScore2(Vector allDocs, Vector matrix, 
			int [] frontierIndex, SearchFaq sf, Hashtable idfHash){

		for(int docnum = 0; docnum < allDocs.size(); docnum++){
			MyDoc md = (MyDoc)allDocs.get(docnum);
			QApair qap = sf.getDocFromSingleId(md.hitId);
			md.setQApair(qap);			
			String question = ((QApair)md.qap).quest;
			question = question.trim();
			question = question.replaceAll("\\b\\s{2,}\\b", " ");
			String [] words = question.split("[ ]+");

			for(int j=0; j<matrix.size(); j++){
				int tFrontierIndex = frontierIndex[j];
				Vector variationSet = (Vector)matrix.get(j);

				while((!md.isIndexSet(j))&&(tFrontierIndex < variationSet.size())){					
					for(int wordnum = 0; wordnum < words.length; wordnum++){
						Variation var = (Variation)variationSet.get(tFrontierIndex);
						if(var.word.equals(words[wordnum])){							
							md.incrementCurrentScore(var.weight);
							md.setIndex(j);
							md.addWord(var.word, var.weight);
							break;
						}
					}
					tFrontierIndex++;
				}
			}	
		}	
		return allDocs;
	}

	public static Vector Union(Vector<MyDoc> allDocs, Vector docs, Variation var, 
			int bitVectorSize){
		if(allDocs == null){
			allDocs = new Vector<MyDoc>();
		}
		for(int i=0; i< docs.size(); i++){
			Integer hitId = (Integer)docs.get(i);
			//Integer hitId = Integer.parseInt(((Token)docs.get(i)).token);
			int flag = 0;
			for(int j=0; j< allDocs.size(); j++){
				MyDoc mydoc = (MyDoc)allDocs.get(j);				
				if(hitId.intValue()== mydoc.hitId){			
					Bit bit = (Bit)mydoc.bitVector.get(var.varBitIndex);
					if(bit.notSet()&&mydoc.checkIfNotAdded(var.word)){
						mydoc.incrementCurrentScore(var.weight);
						mydoc.addWord(var.word,var.weight);						
						bit.setBit();
					}
					flag = 1;
					break;
				}
			}
			if(flag != 1){
				MyDoc md = new MyDoc();
				Vector bitVector = md.initializeVector(bitVectorSize, var.varBitIndex);
				md.newMyDoc(hitId.intValue(), var.weight, bitVector,var.word);
				allDocs.add(md);
			}
		}
		Collections.sort(allDocs, new ScoreComparer());
		if(allDocs == null){
			Debug.print("Union NULL");
		}
		return allDocs;		
	}
	
	public static double getLowerBound(Vector allDocs){
		double maxScore = 0;
		for(int i=0; i<allDocs.size(); i++){
			MyDoc md = (MyDoc)allDocs.get(i);
			if(maxScore < md.currentScore){
				maxScore = md.currentScore;
			}
		}
		return maxScore;
	}

	public static double getUpperBound(Vector matrix, int [] frontierIndex){
		double upperBound = 0;
		for(int i=0; i< matrix.size(); i++){
			Vector wordNvariations = (Vector)matrix.get(i);
			Variation v ;
			if(frontierIndex[i]<wordNvariations.size()){
				v = (Variation)(wordNvariations.get(frontierIndex[i]));				
			}
			else{
				v = (Variation)(wordNvariations.get(frontierIndex[i]-1));
			}
			upperBound += v.weight;
		}
		return upperBound;
	}

	public static int getMaxWeightWordIndex(Vector matrix, 
			int [] frontierIndex){
		double maxWeight = 0 ;
		int index = -1;
		for(int i=0; i< matrix.size(); i++){
			Vector wordNvariations = (Vector)matrix.get(i);
			if(frontierIndex[i] < wordNvariations.size()){
				Variation v = (Variation)(wordNvariations.get(frontierIndex[i]));
				if(v.weight > maxWeight){
					maxWeight = v.weight;
					index = i;
				}
			}
		}
		return index;
	}
	
	public static void getResult1(Vector allDocs){		
		QApair qap = null;
		if(allDocs != null){
			for(int i=0;i<Math.min(allDocs.size(), 5);i++){
				qap = ((MyDoc)allDocs.get(i)).qap;
				System.out.println(qap.quest);
			}
		}
	}
		
	
	public static void getResult(SearchFaq sf, Vector allDocs){
		Vector doclist = sf.getDocsFromIds(allDocs);		
		System.out.println("Total retrieved docs "+doclist.size());
		QApair qap = null;
		if(doclist != null){
			for(int i=0;i<Math.min(doclist.size(), 5);i++){
				qap = (QApair)doclist.get(i);
				System.out.println(qap.quest);
			}
		}
		else{
			System.out.println("document list is empty");
		}
	}
			
	public static void getSingleOptimizedResult(String indexDir,String smsQuery,LCS lcs, Vector dictionary, Hashtable 
			idfHash, String indexdir, SearchFaq sf, TokenFetcher tf){
		Vector matrix = lcs.getMatrix(indexDir,smsQuery, idfHash,dictionary, tf);
		System.out.println("5");
		try {
			Debug.printMatrix(matrix);
			Vector allDocs = getDocuments(matrix,indexdir,idfHash);
			printOptimizedResult(allDocs);
		} catch (UnsupportedEncodingException e) {
			e.printStackTrace();
		}		
	}
	
	public static Vector<MyDoc> normalizeQuestions(Vector<MyDoc> allDocs){
		if(allDocs != null){
			Iterator<MyDoc> it = allDocs.iterator();
			while(it.hasNext()){
				MyDoc md = it.next();
				md.normalize();
			}
		}
		//Collections.sort(allDocs, new ScoreComparer());
		return allDocs;
	}
	public static void getOptimizedResult(String indexDir,String queryFile, String resultFile, LCS lcs, 
			Vector dictionary, Hashtable idfHash, String indexdir, SearchFaq sf, TokenFetcher tf){
		
		Result result = new Result();
		Hashtable<String, String> QA = null;
		if(map){
			QA = getQAHash();	
		}
		String smsQuery = null;
		File file = new File(queryFile);
		
		File writeFile = new File(resultFile);
		try {
			//BufferedReader reader = new BufferedReader(new FileReader(file));
			BufferedReader reader = new BufferedReader(new InputStreamReader (new FileInputStream (file), "UTF8"));
			//BufferedWriter writer = new BufferedWriter(new FileWriter(writeFile, true));
			BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(writeFile, true),"UTF8"));
			int queryNumber = 0;
			boolean relevancy;
			while((smsQuery = reader.readLine())!= null){
				queryNumber++;
				if(queryNumber < 61){
					relevancy = true;
				} else {
					relevancy = false;
					break;
				}
				smsQuery = smsQuery.toLowerCase();
				//System.out.println(smsQuery);
				//long time = System.currentTimeMillis();
				Vector matrix = lcs.getMatrix(indexDir,smsQuery, idfHash,dictionary, tf);
				//long totalTime = System.currentTimeMillis()-time;
				//System.out.println(totalTime);
				
				long time = System.currentTimeMillis();
				Vector allDocs = getDocuments(matrix,indexdir,idfHash);
				long totalPruningTime = System.currentTimeMillis()-time;
				writer.write("Total time :: "+Long.toString(totalPruningTime));
				writer.newLine();
				int size = 0;
				if(allDocs != null){
					size = allDocs.size();
				}
				writer.write("Number of Docs "+ size);
				writer.newLine();
				writer.write(smsQuery);
				System.out.println(smsQuery);
				Debug.print(smsQuery);
				writer.newLine();
				printOptimizedResultinFile(smsQuery, allDocs, QA, writer, result, relevancy);				
				writer.newLine();
				
				/*
				time = System.currentTimeMillis();
				allDocs = naiveGetDocuments(matrix,indexdir);
				Vector doclist = sf.getDocsFromIds(allDocs);
				long totalNaiveTime = System.currentTimeMillis()-time;
				writer.write("Total time :: "+Long.toString(totalNaiveTime));
				writer.newLine();
				writer.write("Number of Docs "+Integer.toString(allDocs.size()));
				writer.newLine();
				writer.write(smsQuery);
				writer.newLine();
				printNaiveResultinFile(doclist, writer);				
				writer.newLine();
*/
				writer.write("------------------------------------------------------------------------------");
				//writer.newLine();
			}
			System.out.println(">>>>>>>>>>>>><<<<<<<<<<<");
			result.printStats2();
			System.out.println(">>>>>>>>>>>>><<<<<<<<<<<");
			reader.close();
			writer.close();
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}	
	 }

	
	public static Vector getDictionary(String unigramfile){
		File file = new File(unigramfile);
		Vector dictionary = new Vector();
		String dicWord = null;
		try {
			BufferedReader reader = new BufferedReader(new FileReader(file));
			while((dicWord = reader.readLine())!= null){				
				dictionary.add(dicWord);
			}		
		} catch (FileNotFoundException e) {
			
			e.printStackTrace();
		} catch (IOException e) {
			
			e.printStackTrace();
		}
		return dictionary;
	}
	
	public static Vector getDictionaryMatrix(String unigramfile){
		File file = new File(unigramfile);
		Vector dictionaryMatrix = new Vector();
		for(int i=0; i< 26; i++){
			Vector dictionary = new Vector();
			dictionaryMatrix.add(dictionary);
		}
		
		String dicWord = null;
		try {
			BufferedReader reader = new BufferedReader(new FileReader(file));
			while((dicWord = reader.readLine())!= null){				
				int index = getAlphabetIndex(dicWord.toLowerCase().charAt(0));
				if(index != -1){
					Vector dictionary = (Vector)dictionaryMatrix.get(index);
					dictionary.add(dicWord);
				}
			}		
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e) {
			
			e.printStackTrace();
		}
		return dictionaryMatrix;
	}
	
	public static int getAlphabetIndex(char alphabet){
		
		if(alphabet == 'a'){return 0;}
		else if(alphabet == 'b'){return 1;}
		else if(alphabet == 'c'){return 2;}
		else if(alphabet == 'd'){return 3;}
		else if(alphabet == 'e'){return 4;}
		else if(alphabet == 'f'){return 5;}
		else if(alphabet == 'g'){return 6;}
		else if(alphabet == 'h'){return 7;}
		else if(alphabet == 'i'){return 8;}
		else if(alphabet == 'j'){return 9;}
		else if(alphabet == 'k'){return 10;}
		else if(alphabet == 'l'){return 11;}
		else if(alphabet == 'm'){return 12;}
		else if(alphabet == 'n'){return 13;}
		else if(alphabet == 'o'){return 14;}
		else if(alphabet == 'p'){return 15;}
		else if(alphabet == 'q'){return 16;}
		else if(alphabet == 'r'){return 17;}
		else if(alphabet == 's'){return 18;}
		else if(alphabet == 't'){return 19;}
		else if(alphabet == 'u'){return 20;}
		else if(alphabet == 'v'){return 21;}
		else if(alphabet == 'w'){return 22;}
		else if(alphabet == 'x'){return 23;}
		else if(alphabet == 'y'){return 24;}
		else if(alphabet == 'z'){return 25;}
		else {return -1;}
	}
	
	public static Hashtable<String,String> getQAHash(){
		Hashtable<String,String> OrigQA = new Hashtable<String,String>();
		
		String queryAnswerFile = 
			"C:/DOCUME~1/Administrator/workspace/smsindex/data/queryAnswerMap.txt";
		String queryFile = SMSConstant.SMS_hindiFAQ;
		//file containing query
		try {
			BufferedReader queryReader = new BufferedReader(new InputStreamReader 
					(new FileInputStream (queryFile), "UTF8"));
			//file containing hindi answer
			BufferedReader hindiAnswerReader = new BufferedReader(new InputStreamReader 
					(new FileInputStream (queryAnswerFile), "UTF8"));
			String  rightAn = null;
			Vector<String> rightAns = new Vector<String>();
			while((rightAn = hindiAnswerReader.readLine())!= null){
				rightAns.add(rightAn);
			}
			
			Iterator <String> it = rightAns.iterator();
			String query = null;
			while((query = queryReader.readLine())!= null){
				if(it.hasNext()){
					String nextS = it.next();
					OrigQA.put(query.toLowerCase(), nextS);
				}else{
					OrigQA.put(query.toLowerCase(), "NON RELEVANT QUERY");
				}
			}
			queryReader.close();
			hindiAnswerReader.close();
			
		} catch (UnsupportedEncodingException e) {
			e.printStackTrace();
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
		return OrigQA;
	}

	
	
		
	public static void printOptimizedResult(Vector allDocs)
	throws UnsupportedEncodingException{
		QApair qap = null;
			PrintStream sysout = 
				new PrintStream(System.out, true, "UTF-8");
			if(allDocs != null){
				for(int i=0;i<Math.min(allDocs.size(),20);i++){
					MyDoc md =(MyDoc)allDocs.get(i);
					qap = md.qap;
					    System.out.println(md.currentScore);
						sysout.println(qap.quest);
						sysout.println(md.wholeSent);
				}
			} else{
				System.out.println("NULL DOCS RETURNED");
			}
		} 	
	
	public static void printOptimizedResultinFile(
			String smsQuery, Vector allDocs, Hashtable<String, String> 
			QA, BufferedWriter writer,
			Result result, boolean relevancy){
			
		QApair qap = null;
		
		double threshold = 8;
		double previousScore = 0;
		if(allDocs != null){
				for(int i=0;i<allDocs.size();i++){
					qap = ((MyDoc)allDocs.get(i)).qap;
					
					try {
						double score = ((MyDoc)allDocs.get(i)).currentScore;
						if(i==0){
							previousScore = score;
						}
						if((score > threshold)&&(score == previousScore)){
							writer.write(qap.quest+"\n SCORE:: "+score+"\n");
							writer.newLine();
						} else{
							break;
						}
					} catch (IOException e) {			
						e.printStackTrace();
					}
				}
			}
			if(map){
				calculateResult(smsQuery, result, allDocs, QA, relevancy, threshold);
				calculateMRR(smsQuery, result, allDocs, QA, relevancy, threshold);
				result.printStats();
			}
			//writer2.close();		
	}

	public static void 
	calculateResult(String smsQuery, Result result, Vector<MyDoc> allDocs,
			Hashtable<String, String> OrigQA, boolean relevancy, double threshold){
		
		//tp,fn,fp,tn
		
		//add using file
		//we have hash where query and answer is there
		String rightAns = OrigQA.get(smsQuery.toLowerCase());
		int tp = 0,fn = 0, tn = 0, fp = 0;
		if(relevancy){
			if(allDocs == null){
				fp++;
			}else{
				
				
				boolean flag = false;				
				if((((MyDoc)allDocs.get(0)).currentScore > threshold)){
						//&&(rightAns.equals(qap.quest))){
					//if((((MyDoc)allDocs.get(0)).currentScore < 11)&&((((MyDoc)allDocs.get(0)).currentScore > 9))){
					double highestScore = ((MyDoc)allDocs.get(0)).currentScore;
					for(int i=0; i< allDocs.size(); i++){
						QApair qap = (allDocs.get(i)).qap;						
						double score = ((MyDoc)allDocs.get(i)).currentScore;
						if(score != highestScore){
							break;
						}
						if(rightAns.equals(qap.quest)){
							flag = true;
							tp++;
							break;
						}
						System.out.println("Score ::" + ((MyDoc)allDocs.get(0)).currentScore);
						System.out.print("Query :: ");
						Debug.printUTF(smsQuery);
						System.out.print("Output :: ");
						Debug.printUTF(qap.quest);
						System.out.print("Correct one :: ");
						Debug.printUTF(rightAns);
					}
					if(!flag){
						fp++;
					}
					//tp++;
				
				}else{
					fp++;
				}
			}			
		}else{
			if(allDocs == null){ 
				tn++;
			}else{
				if(((MyDoc)allDocs.get(0)).currentScore > threshold){
					fn++;
				}else{
					tn++;
				}
			}
		}		
		result.update(tp, fp, tn, fn);
	}

	public static void calculateMRR(String smsQuery, Result result, Vector<MyDoc> allDocs,
			Hashtable<String, String> OrigQA, boolean relevancy, double threshold){
		
		String rightAns = OrigQA.get(smsQuery.toLowerCase());
		if(relevancy){
			if(allDocs == null){
				result.update(0);
			}else{
				boolean flag = false;				
				for(int i=0; i< allDocs.size(); i++){
					QApair qap = (allDocs.get(i)).qap;

					if(rightAns.equals(qap.quest)){
						flag = true;
						System.out.println(i+"??????????????");	  
						result.update(1/(new Double(i+1)));
						System.out.println(i+"%%%%%%%%%%%%%%");
						break;
					}
				}
				if(!flag){
					result.update(0);
				}				
			}
		}			
	}
	
	public static void printNaiveResultinFile(Vector doclist, 
			BufferedWriter writer){
				
		for(int i=0; i< Math.min(doclist.size(), 5); i++){
			QApair qap = (QApair)doclist.get(i);
			try {
				writer.write(qap.quest);
				writer.newLine();
			} catch (IOException e) {				
				e.printStackTrace();
			}
		}
	}
	
	

	public static void main(String[] args) {
/*	
		String indexdir = LuceneConstants.yahooindexdir;
		String idfile = SMSConstant.yahootfidf;
		String dicFile = SMSConstant.yahoounigram_onlyidf;		
		String queryFile = SMSConstant.yahooQueryFile;
		String resultFile = SMSConstant.yahooResultFile;
*/
		
/*		
		String indexdir = LuceneConstants.faqindexdir;//
		String idfile = SMSConstant.mobiletfidf;//
		String dicFile = SMSConstant.mobileDic;//		
		//String queryFile = SMSConstant.SMSqueryFile;//		
		//String resultFile = SMSConstant.SMSresultFile;
		String queryFile = SMSConstant.HindiSMSqueryFile;//		
		String resultFile = SMSConstant.HindiSMSresultFile;
*/
		String indexdir = LuceneConstants.hindiFAQIndexdir_DanishData;
		String idfile = SMSConstant.hindiFAQtfidf;//hindi  words
		String dicFile = SMSConstant.englishDic;	//English vocab file
		String queryFile = SMSConstant.SMS_hindiFAQ;
		String resultFile = SMSConstant.Result_hindiFAQ;
		
		String smsQuery = "hw shll i contct irctc";
		LCS lcs = new LCS();
		TokenFetcher tf = new TokenFetcher();
		try {
			TokenFetcher.LoadMap(TokenFetcher.lexFile.replace("\"", ""));
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (Exception e) {
			e.printStackTrace();
		}
		File file = new File(idfile);
		System.out.println("1");
		Hashtable idfHash = lcs.getIdfHash(file);//hindi idf scores
		System.out.println("2");
		SearchFaq sf = new SearchFaq(indexdir);
		System.out.println("3");
		Vector dictionary = getDictionaryMatrix(dicFile);
		System.out.println("4");
		//getSingleOptimizedResult(indexdir,smsQuery.toLowerCase(), 
			//	lcs,dictionary,idfHash,indexdir, sf, tf);
		getOptimizedResult(indexdir,queryFile, resultFile, lcs, dictionary,
				idfHash, indexdir, sf, tf);
	}
}